{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So After looking at the answer of final, I modify (or correct) of my final homework\n",
    "# with method of categoricals and several parameters and also for importance, I changed to get_fscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from bayes_opt import bayesian_optimization\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train dataset shape is: (9674, 15)\n",
      "test dataset shape is: (11182, 15)\n"
     ]
    }
   ],
   "source": [
    "# question 1: read adult data.txt as train, adult test.txt as test, rename their headers to ['age', \n",
    "# 'workclass', 'fnlwgt', 'education', 'education_num', 'marital_status', 'occupation', 'relationship',\n",
    "# 'race', 'sex', 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country', 'wage_class']\n",
    "column_name = ['age', 'workclass', 'fnlwgt', 'education', 'education_num', 'marital_status', \n",
    "               'occupation', 'relationship', 'race', 'sex', 'capital_gain', 'capital_loss', \n",
    "               'hours_per_week', 'native_country', 'wage_class']\n",
    "df_train = pd.read_csv('adult+data.txt', header = None)\n",
    "df_test = pd.read_csv('adult+test.txt', header = None)\n",
    "df_train.columns = column_name\n",
    "df_test.columns = column_name\n",
    "print(\"train dataset shape is: \" + str(df_train.shape))\n",
    "print(\"test dataset shape is: \" + str(df_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train dataset shape without '?' is: (8936, 15)\n",
      "test dataset shape without '?' is: (10347, 15)\n"
     ]
    }
   ],
   "source": [
    "# question 2: replace ‘ ?’ with nan and drop na\n",
    "df_train_nona = df_train.replace(' ?', np.nan).dropna()\n",
    "df_test_nona = df_test.replace(' ?', np.nan).dropna()\n",
    "print(\"train dataset shape without '?' is: \" + str(df_train_nona.shape))\n",
    "print(\"test dataset shape without '?' is: \" + str(df_test_nona.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# question 3: replace test wage_class all of the ‘<=50K.’ \n",
    "# with ‘<=50K’ and the same for ‘>50K.’ with ‘>50K’, same as train\n",
    "df_test_rep = df_test_nona.replace([' >50K.', ' <=50K.'], ['>50K', '<=50K'])\n",
    "df_train_rep = df_train_nona.replace([' >50K', ' <=50K'], ['>50K', '<=50K'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# question 4: apply Ordinal Encoding to Categoricals for both train and test\n",
    "df_combined = pd.concat([df_test_rep, df_train_rep], axis = 0)\n",
    "for i in df_combined.columns:\n",
    "    if df_combined[i].dtype == 'object': \n",
    "        df_combined[i] = pd.Categorical(df_combined[i]).codes\n",
    "df_test_final = df_combined[:df_test_rep.shape[0]]\n",
    "df_train_final = df_combined[df_test_rep.shape[0]:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pop is better and easier way\n",
    "y_train = df_train_final['wage_class']\n",
    "y_test = df_test_final['wage_class']\n",
    "x_train = df_train_final.drop(columns = 'wage_class')\n",
    "x_test = df_test_final.drop(columns = 'wage_class')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | colsam... |   gamma   | learni... | max_depth | min_ch... | n_esti... | subsample |\n",
      "-------------------------------------------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.9171  \u001b[0m | \u001b[0m 0.8049  \u001b[0m | \u001b[0m 0.9677  \u001b[0m | \u001b[0m 0.02242 \u001b[0m | \u001b[0m 5.482   \u001b[0m | \u001b[0m 2.619   \u001b[0m | \u001b[0m 1.357e+0\u001b[0m | \u001b[0m 0.8222  \u001b[0m |\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m 0.913   \u001b[0m | \u001b[0m 0.7914  \u001b[0m | \u001b[0m 0.7382  \u001b[0m | \u001b[0m 0.06384 \u001b[0m | \u001b[0m 3.73    \u001b[0m | \u001b[0m 1.07    \u001b[0m | \u001b[0m 1.832e+0\u001b[0m | \u001b[0m 0.8335  \u001b[0m |\n",
      "| \u001b[95m 3       \u001b[0m | \u001b[95m 0.9176  \u001b[0m | \u001b[95m 0.8514  \u001b[0m | \u001b[95m 0.4172  \u001b[0m | \u001b[95m 0.0372  \u001b[0m | \u001b[95m 4.214   \u001b[0m | \u001b[95m 1.403   \u001b[0m | \u001b[95m 1e+03   \u001b[0m | \u001b[95m 0.7477  \u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 0.9107  \u001b[0m | \u001b[0m 0.7683  \u001b[0m | \u001b[0m 0.9391  \u001b[0m | \u001b[0m 0.06017 \u001b[0m | \u001b[0m 5.96    \u001b[0m | \u001b[0m 4.483   \u001b[0m | \u001b[0m 1e+03   \u001b[0m | \u001b[0m 0.7749  \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 0.9074  \u001b[0m | \u001b[0m 0.761   \u001b[0m | \u001b[0m 0.436   \u001b[0m | \u001b[0m 0.09803 \u001b[0m | \u001b[0m 3.553   \u001b[0m | \u001b[0m 1.155   \u001b[0m | \u001b[0m 2e+03   \u001b[0m | \u001b[0m 0.7186  \u001b[0m |\n",
      "| \u001b[95m 6       \u001b[0m | \u001b[95m 0.9202  \u001b[0m | \u001b[95m 0.8893  \u001b[0m | \u001b[95m 0.328   \u001b[0m | \u001b[95m 0.02215 \u001b[0m | \u001b[95m 3.027   \u001b[0m | \u001b[95m 1.013   \u001b[0m | \u001b[95m 1.577e+0\u001b[0m | \u001b[95m 0.8668  \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 0.9136  \u001b[0m | \u001b[0m 0.784   \u001b[0m | \u001b[0m 0.2693  \u001b[0m | \u001b[0m 0.08585 \u001b[0m | \u001b[0m 3.105   \u001b[0m | \u001b[0m 1.03    \u001b[0m | \u001b[0m 1.191e+0\u001b[0m | \u001b[0m 0.8346  \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 0.9114  \u001b[0m | \u001b[0m 0.7151  \u001b[0m | \u001b[0m 0.03935 \u001b[0m | \u001b[0m 0.09402 \u001b[0m | \u001b[0m 3.103   \u001b[0m | \u001b[0m 1.074   \u001b[0m | \u001b[0m 1.48e+03\u001b[0m | \u001b[0m 0.8382  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 0.8994  \u001b[0m | \u001b[0m 0.7606  \u001b[0m | \u001b[0m 0.1962  \u001b[0m | \u001b[0m 0.09092 \u001b[0m | \u001b[0m 6.751   \u001b[0m | \u001b[0m 1.044   \u001b[0m | \u001b[0m 1.698e+0\u001b[0m | \u001b[0m 0.8181  \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 0.9149  \u001b[0m | \u001b[0m 0.7705  \u001b[0m | \u001b[0m 0.6094  \u001b[0m | \u001b[0m 0.03936 \u001b[0m | \u001b[0m 3.03    \u001b[0m | \u001b[0m 4.973   \u001b[0m | \u001b[0m 1.917e+0\u001b[0m | \u001b[0m 0.7078  \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 0.9195  \u001b[0m | \u001b[0m 0.7322  \u001b[0m | \u001b[0m 0.6297  \u001b[0m | \u001b[0m 0.03822 \u001b[0m | \u001b[0m 3.126   \u001b[0m | \u001b[0m 1.258   \u001b[0m | \u001b[0m 1.287e+0\u001b[0m | \u001b[0m 0.8465  \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 0.914   \u001b[0m | \u001b[0m 0.8292  \u001b[0m | \u001b[0m 0.2261  \u001b[0m | \u001b[0m 0.09245 \u001b[0m | \u001b[0m 3.076   \u001b[0m | \u001b[0m 1.077   \u001b[0m | \u001b[0m 1.069e+0\u001b[0m | \u001b[0m 0.8976  \u001b[0m |\n",
      "=============================================================================================================\n",
      "-----------------------------------------------------\n",
      "Final Results\n",
      "XGBOOST: 0.920186\n"
     ]
    }
   ],
   "source": [
    "# question 5: build xgboost: wage_class is the response variable, and others are predictors\n",
    "# use bayesian to find the best combination of tunning parameters:\n",
    "# 'max_depth': (3,7) 'min_child_weight': (1,5), 'learning_rate': (0.01, 0.1), 'subsample': (0.7,0.9)\n",
    "# 'colsample_bytree': (0.7,0.9), 'n_estimators': (1000, 2000), ' 'gamma': (0.01,1.)\n",
    "def xgboostcv(max_depth, learning_rate, n_estimators, gamma, min_child_weight, \n",
    "              subsample, colsample_bytree, silent = True, nthread = -1):\n",
    "    return cross_val_score(xgb.XGBClassifier(max_depth = int(max_depth), learning_rate = learning_rate, \n",
    "                                             n_estimators = int(n_estimators), silent = silent, nthread = nthread, \n",
    "                                             gamma = gamma, min_child_weight = min_child_weight, subsample = subsample, \n",
    "                                             colsample_bytree = colsample_bytree),\n",
    "                           x_train, \n",
    "                           y_train, \n",
    "                           scoring = 'roc_auc', \n",
    "                           cv=5).mean()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    xgboostBO = bayesian_optimization.BayesianOptimization(xgboostcv,\n",
    "                                 {'max_depth': (3, 7),\n",
    "                                  'learning_rate': (0.01, 0.1),\n",
    "                                  'n_estimators': (1000, 2000),\n",
    "                                  'gamma': (0.01, 1.),\n",
    "                                  'min_child_weight': (1, 5),\n",
    "                                  'subsample': (0.7, 0.9),\n",
    "                                  'colsample_bytree' :(0.7, 0.9)})\n",
    "    xgboostBO.maximize(init_points=2, n_iter = 10)\n",
    "    print('-'*53)\n",
    "    print('Final Results')\n",
    "    temp = xgboostBO.res\n",
    "    temp_list = []\n",
    "    for i in temp:\n",
    "        temp_list.append(i['target'])\n",
    "    print('XGBOOST: %f' % max(temp_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train-error-mean</th>\n",
       "      <th>train-error-std</th>\n",
       "      <th>test-error-mean</th>\n",
       "      <th>test-error-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.166126</td>\n",
       "      <td>0.002137</td>\n",
       "      <td>0.171329</td>\n",
       "      <td>0.009131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.165678</td>\n",
       "      <td>0.002252</td>\n",
       "      <td>0.170545</td>\n",
       "      <td>0.008220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.166769</td>\n",
       "      <td>0.002579</td>\n",
       "      <td>0.171777</td>\n",
       "      <td>0.009728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.163328</td>\n",
       "      <td>0.003388</td>\n",
       "      <td>0.165845</td>\n",
       "      <td>0.009128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.163216</td>\n",
       "      <td>0.003315</td>\n",
       "      <td>0.164949</td>\n",
       "      <td>0.009831</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train-error-mean  train-error-std  test-error-mean  test-error-std\n",
       "0          0.166126         0.002137         0.171329        0.009131\n",
       "1          0.165678         0.002252         0.170545        0.008220\n",
       "2          0.166769         0.002579         0.171777        0.009728\n",
       "3          0.163328         0.003388         0.165845        0.009128\n",
       "4          0.163216         0.003315         0.164949        0.009831"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# question 6: use xgb.cv to find the early stopping rounds based on error\n",
    "# parameter is from the max score above\n",
    "train_initial = xgb.DMatrix(x_train, y_train)\n",
    "target_para = {}\n",
    "for i in temp:\n",
    "    if i['target'] == max(temp_list):\n",
    "        target_para = i\n",
    "param = {'eta':  0.0159, 'seed':0, 'subsample': target_para['params']['subsample'], \n",
    "         'colsample_bytree': target_para['params']['colsample_bytree'], 'objective': 'binary:logistic', \n",
    "         'max_depth': int(target_para['params']['max_depth'].round()), \n",
    "         'min_child_weight': target_para['params']['min_child_weight'], \n",
    "         'gamma': target_para['params']['gamma']}\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "df_cv = xgb.cv(params = param, dtrain = train_initial, num_boost_round = 2000, \n",
    "               nfold = 5, metrics = ['error'], early_stopping_rounds = 100)\n",
    "df_cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb0AAAEICAYAAADLKSqCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3debxd873/8ddbEHOM9QjFMQRNDCmJWaWkWlpDW5qqKbiUa2wvpaUa/JTSW9d0abjEVDSGcmnNCRGRSWZzJS4xFyHGiM/vj/Xdycq2z845J+fsvU/W+/l4nMdZe63v+q7vXmfHx3ftvddbEYGZmVkRLFHvAZiZmdWKi56ZmRWGi56ZmRWGi56ZmRWGi56ZmRWGi56ZmRWGi55Zg5E0TVK/eo/DbHEkf0/PzCqRNAP4t4h4qN5jMWsvnumZ2QIkLVnvMZh1FBc9swYjaYak/pIGSRoq6UZJH0qaImljSb+W9JakVyTtnttvuKTzJI2R9IGkuyStmtu+d7p0+n5q+42yY54qaTLwkaSbgXWB/5U0W9KvUruhkt6QNEvSY5J65foYIulySfem8Y6WtGFuey9JD0p6V9Kbkn6T1i8h6TRJ/5T0L0l/zY/brD256Jk1tr2AG4BVgAnA/WT/btcGzgb+XNb+EOBwoDvwBXAJgKSNgZuBk4A1gL+TFbSlc/seAHwfWDkiDgD+D9grIlaIiAtSm38APYCvAU8BN5Ud/6fAWWm8LwLnpuOvCDwE3AesBWwEPJz2OR7YF9glbXsPuLzlp8is5Vz0zBrbiIi4PyK+AIaSFazzI2IOcAvQJGnlXPsbImJqRHwE/Bb4iaQuwADg3oh4MO37R2BZYIfcvpdExCsR8Ulzg4mIayLiw4j4DBgEbCmpW67JnRExJo33JqB3Wv8D4I2I+M+I+DT1MTptOxo4PSJezfW7ny+zWkdw0TNrbG/mlj8B3omIubnHACvk2rySW34ZWApYnWwG9XJpQ0R8mdqu3cy+XyGpi6Tz02XID4AZadPquWZv5JY/zo1tHeCfzXS9HnBnuuz6PvAMMBdYs9p4zNrCRc9s8bJObnldYA7wDvAaWXEBQJJS25m59uUf5S5//DNgH6A/0A1oKnXXgnG9AmxQZdseEbFy7meZiJjZTHuzNnPRM1u8HCSpp6TlyN7zuy3NDP8KfF/SbpKWAv4D+Ax4okpfb7JgoVox7fMvYDng960Y1z1Ad0knSeoqaUVJ26ZtVwLnSloPQNIakvZpRd9mLeaiZ7Z4uQEYQnaZcRngBICIeA44CLiUbOa3F9mHVD6v0td5wBnpsuPJwPVkl0hnAk8DT7Z0UBHxIfCddNw3gBeAb6fNFwN3Aw9I+jD1u22lfswWlb+cbraYkDQcuDEirq73WMwalWd6ZmZWGC56ZmZWGL68aWZmheGZnpmZFYbveFBHq6++ejQ1NdV7GGZmncr48ePfiYg12rKvi14dNTU1MW7cuHoPw8ysU5H08sJbVebLm2ZmVhguemZmVhi+vFlHU2bOoum0e+s9DDOzmppx/vfrdmzP9MzMrDBc9MzMrDBc9KqQ9DdJ4yVNk3RUWneEpOcljZF0laTL0vo1JN0uaWz62bG+ozczs3J+T6+6wyPiXUnLAmMl3UuWRr0V8CHwCDAptb0YuCgiHpe0LnA/8I3yDlPxPAqgy0pt+pqJmZm1kYtedSdI+mFaXgc4GHg0It4FkDQU2Dht7w/0zLI5AVhJ0goRMTvfYUQMBgYDdO3ew/eAMzOrIRe9ZkjqR1bIto+Ij1Nsy7NUmL0lSwDbRcSntRmhmZm1lt/Ta1434L1U8DYFtgOWB3aRtIqkJYEf59o/ABxfeiCpd01Ha2ZmC+WZXvPuA46W9AzwHFma80zg98AY4F2ymd+s1P4E4HJJk8nO62PA0dUOsPna3RhXx++rmJkVjYteMyLiM2CP8vWSxkXE4DTTuxP4W2r/DjCgtqM0M7PW8OXN1hskaSIwFZhOKnpmZtb4PNNrpYg4ud5jMDOztvFMz8zMCsNFz8zMCsNFz8zMCsNFz8zMCsMfZKkj5+mZWaOpZ9ZdLXTKmZ6ktSTdlpZ7S9qzBfv0k3RPO4/j75JWbs8+zcys43TKohcRr0XEfulhb2ChRa+DxrFnRLxfj2ObmVnr1aXoSTpE0mRJkyTdIGkvSaMlTZD0kKQ1U7tBafsoSS9IOjKtb5I0VdLSwNnAAEkTJQ2QtE1qP0HSE5I2aeGY1pD0YMrOu1rSy5JWT9u+kquX1s+QtHoazzMpX2+apAdSHJGZmTWQmhc9Sb2AM4BdI2JL4ETgcbKEgm8CtwC/yu2yBbArsD1wpqS1Shsi4nPgTODWiOgdEbeS3Q9z59TXmWT3ymyJ3wGPREQv4DZg3dy2wyNia6APWdzQahX27wFcnvZ/nwVvRp1//kdJGidp3NyPZ1VqYmZmHaQeH2TZFRia7lVJCmndHLhVUndgabLbe5XcFRGfAJ9IGgZsA0ys0n834DpJPYAAlmrhuHYCfpjGdJ+k93LbynP1egD/Ktt/ekSUxjUeaKp0EOfpmZnVT6O8p3cpcFlEbA78HFgmt628MCysUJwDDIuIzYC9yvpqtbJcvS2BCc30+VlueS7+ZKyZWcOpR9F7BNi/dIlQ0qpks7OZafuhZe33kbRMat8PGFu2/UNgxdzjfF8DWzGukcBP0ph2B1bJ9Veeq2dmZp1QzWcjETFN0rnAo5Lmks2cBgFD0yXFR4D1c7tMBoYBqwPnRMRrkppy24cBp6Xkg/OAC8gub54BtOZLcGcBN0s6GBgFvEFWUCvl6rUL5+mZmdWWIhr3bSVJg4DZEfHHGhyrKzA3Ir6QtD1wRUR0aPp5nz59Yty4cR15CDOzxY6k8RHRpy37+n2n+dYF/ippCeBz4Mg6j8fMzNpZQxe9iBjU3n1KOozsaxJ5IyPiWOCb7X08MzNrHA1d9DpCRFwLXFvvcZiZWe01ylcWzMzMOpyLnpmZFUbhLm82EkcLmVkjWNzjhPIKM9OTdEK6KfRNVdrMbofjDMzfH9TMzBpHkWZ6/w70j4hXO/g4A4GpwGsdfBwzM2ulQsz0JF0JbAD8Q9IsSddIGi7pJUknVGh/uaS90/Kdkq5Jy4enu8kg6beSnpP0uKSbJZ0saT+yJIabUtSR44XMzBpIIYpeRBxNNvP6NnARsCnwXbLEht9JKk9iGAHsnJbXBnqm5Z2BxyT1JYsO2hLYg6zQERG3AeOAA1PU0SflY3G0kJlZ/RSi6FVwb0R8luKN3gLWLNs+AthZUk/gaeDNFHu0PfAEsCNZ5NGnEfEh8L8tPXBEDI6IPhHRp8ty3drlyZiZWcsU6T29vKoxQBExU9LKwPeAx4BVyRIYZkfEh5JqNlAzM2s/RZ3ptcSTwElkRW8EcHL6DVkM0V4p8mgF4Ae5/cqjjszMrEEUdabXEiOA3SPiRUkvk832RgBExFhJd5PFHr0JTAFKb9ANAa6U9AlZ8OxX3tcrcbSQmVltNXS0UCOTtEJEzJa0HNls8KiIeKo1fThayMys9RwtVB+D0wddlgGua23BMzOz2nPRa6OI+Fm9x2BmZq3jD7KYmVlhuOiZmVlhuOiZmVlhuOiZmVlh+IMsdeQ8PbPGUqRcuaLyTK8CSb0l7Zl7vLek0+o5JjMzW3QuepX1BuYVvYi4OyLOr+N4zMysHTRU0ZP0S0lT089Jad0hkiZLmiTphrRuzZRzNyn97CCpSdLUXF8nSxqUlodLujhl3E2VtE1av42kUZImSHpC0iaSlgbOBgak9gNSGvplaZ8mSY+kMT0sad20foikS1I/L6VsPTMzayAN856epK2Bw4BtAQGjJY0FzgB2iIh3JK2aml8CPBoRP5TUBVgBWGUhh1guInpL+hZwDbAZ8Cywc0R8Iak/8PuI+LGkM4E+EXFcGtvAXD+Xkt2B5TpJh6ex7Ju2dQd2Isvruxu4rcLzPAo4CqDLSmu09PSYmVk7aJiiR1Ys7oyIjwAk3UEWzjo05d4REe+mtrsCh6R1c4FZkhZW9G5O7R+TtFKKDloRuE5SDyCA8jDZSrYHfpSWbwAuyG37W0R8CTwtqTyjj3T8wcBggK7de/jGp2ZmNdRQlzcX0Rcs+HyWKdteXmACOAcYFhGbAXtV2Ke18jl9Dt0zM2swjVT0RgD7SlpO0vLAD4FxwP6SVgPIXd58GDgmresiqRtZxM/XJK0mqSsLZtwBDEjtdwJmRcQsoBswM20fmGtbLRPvCeCnaflA5mfsmZlZg2uYy5sR8ZSkIcCYtOrqiBgp6VzgUUlzgQlkxelEspSDI8iSz4+JiFGSzk77zyR7vy7vU0kTyC5hHp7WXUB2efMMIP+FuWHAaZImAueV9XM8cK2kU4C3yd6HbBPn6ZmZ1VYh8vQkDQdOjoiGCq9znp6ZWestSp5eI13eNDMz61ANc3mzI0VEv3qPwczM6s8zPTMzKwwXPTMzKwwXPTMzK4xCvKfXqBwtZNY6jv6xReWZnpmZFUanLnqS1pJ0W1peIAOvyj79JN1TZfu8RAUzM1u8dOqiFxGvRUQpwmeBDDwzM7NydS165Vl5kvaSNDrl2z1USiqQNChtHyXpBUlHpvVNKR+vUgbeV7Ly2jC+5rLz9k/HnSTpsbSul6Qx6fiTU3JDpT6PkjRO0ri5H89q66kzM7M2qNsHWST14qtZeQFsFxEh6d+AXwH/kXbZAtgOWB6YIGneJ0Ai4vMKGXgrUZaVB/y4lcNsLjvvTOC7ETEzRRQBHA1cHBE3pSLcpVKHjhYyM6ufen56c1fKsvIkbQ7cKqk7sDQwPdf+roj4BPhE0jBgG2Bilf670fqsvHLNZeeNBIZI+itwR1o3Cjhd0teBOyLihTYcz8zMOlCjvad3KXBZRGwO/JwF8+0q5eFV095ZefMPHHE02Sx1HWC8pNUi4i/A3sAnwN8l7dpexzMzs/ZRz5neI8Cdkv4UEf9Klzfz+XaHlrXfR9J5ZJc3+wGnkc0GS8oz8JrLymuNUnbeDeSy8yRtGBGjgdGS9gDWSZl+L0XEJem9vy3Sc2yWo4XMzGqrbjO9iJgGlLLyJgF/AgYBQyWNB94p22UyWc7dk8A5EfFa2fZhQM/SB1nILkWelzL02lrcjwcOkzQZOJgsxw/gQklTJE0lK4yTgJ8AU1MG32bA9W08ppmZdZBOkacnaRAwOyL+WO+xtCfn6ZmZtZ7z9MzMzFqgU9x7MyIGtXefkg5j/uXKkpERcWx7H8vMzBpDpyh6HSEirgWurfc4zMysdnx508zMCsNFz8zMCqOwlzcbgfP0zBbOGXrWnjzTMzOzwmj3otcReXSS9pXUM/f47HQTaTMzsxbrLDO9fYF5RS8izoyIh+o4HjMz64RaXfQkHZTLjfuzpC6SDpP0vKQxwI65tkMk7Zd7PDu3fGq6ldckSeendUdKGpvW3S5pOUk7kN3I+cJ0zA3z/UraLWXmTZF0jaSuaf0MSWdJeipt27TKcxqU9h0u6SVJJ6T1TelWY6V2J6e7w5DaXpSy8Z6R1FfSHSnv7/9VOZbz9MzM6qRVRU/SN4ABwI4R0RuYCxwEnEVW7HYiNyOr0s8ewD7AthGxJfMje+6IiL5p3TPAERHxBHA3cEpE9I6If+b6WQYYAgxIyQxLAsfkDvVORGwFXAGcvJBhbQp8lyyy6HeSWhJF9Hm6Fc6VwF3AsWT33RwoabVKO0TE4IjoExF9uizXrQWHMDOz9tLamd5uwNbA2HRj5d2AXwDDI+LtiPgcuLUF/fQHro2IjyHL0kvrN5M0QtIUslSDXgvpZxNgekQ8nx5fB3wrt72UdTceaFpIX/dGxGcp3+8tYM0WPI+70+8pwLSIeD0iPgNeIosdMjOzBtLaoieyJPHe6WcTsmSE5nxROoakJVgwCqiSIcBxadZ2FouegfdZ+j2XhX8947Pccqn9vPEn5eMp7fNl2f5ftuB4ZmZWY639D/PDwF2SLoqIt1IG3gTg4nQ57wNgf7KoHYAZZDPDv5K9L1e6ZPggcKakmyLiY0mrptneisDr6dLigczPwyvPyit5DmiStFFEvEgW//NoK59TNW8CX0vPbTbwA+C+9urceXpmZrXVqpleRDxNlhj+QMqYexDoTjbbGwWMJHsvruQqYJeUl7c98FHq5z6yS4Pj0mXS0vttvwVGp36ezfVzC3BK+sDKhrnxfAocRpbBN4VshnVla57TQp7vHOBsYEx6rs9W38PMzBpZp8jTW1w5T8/MrPWcp2dmZtYChfqwhTP0zMyKrVBFzxl6ZmbF5subZmZWGIWa6TUaRwtZ0TgmyOrNMz0zMyuMhRa98psuW2U+T2Zmja8uMz1JNbusWstjmZlZY2tp0esi6SpJ0yQ9IGlZSb0lPSlpsqQ7Ja0C8yJ3+qTl1SXNSMsDJd0t6RHgYUndJT2W4oKmStq5uYNLmp1ifKZJeljSGmn9hpLukzQ+3ah607R+iKQrJY1mfoJDeZ9TJK2szL8kHZLWXy/pOyky6cIUdTRZ0s9z+56SW39Whb43SHeP6Vthm6OFzMzqpKVFrwdweUT0At4HfgxcD5waEVuQpQz8rgX9bAXsFxG7AD8D7k8RRVsCE6vstzwwLh3/0dyxBgPHR8TWZLcy++/cPl8HdoiIXzbT50iyOKReZKkIpaK7PfAEcAQwKyL6An2BIyWtL2n3dD62AXoDW0ual+wgaRPgdmBgRIwtP6ijhczM6qell/6mR0SpKI0HNgRWjojSzZ2vA4a2oJ8HczFCY4Fr0s2l/5brv5IvmR9ZdCNwh6QVgB3I7rtZatc1t8/QiJhbpc8RZDFEL5Pl7R0laW3gvYj4KBW3LTQ/BLcbWbHbPf1MSOtXSOv/D1iDLFfvR+k+pWZm1kBaOtMrj91ZuUrbfBxPeRTPR6WFiHiMrOjMBIaULi+2UKRjvJ+LOeodEd+odKxmPEY2u9sZGA68DexHVgwhi1E6Ptf3+hHxQFp/Xm79RhHxP2mfWWTFb6dWPBczM6uRtn7IYxbwnqSdI2IEC0b6zCCLExpDVkQqkrQe8GpEXCWpK9mlz+ubab5E6usWssuij0fEB5KmS9o/IoYqm+5tERGTmuljARHxiqTVgaUj4iVJj5NdIj0uNbkfOEbSIxExR9LGZAX6fuCcFIs0O80O56R9Pgd+CNwvaXZE/KXaGBwtZGZWW4vyycZDgSslLUf2nthhaf0fgb9KOgqo9s3rfmRxQXPIsuqqzfQ+AraRdAZZqvmAtP5A4Iq0fimyotiiopeMBrqk5RHAecDj6fHVZGnrT6WC+jawb0Q8IOkbwKh0WXU2cBDZDJh0afQHwIOp8JXS1c3MrM46RbRQKh4r1Hsc7c3RQmZmredoITMzsxZoqC9up+/VdS1bffCizPIcJ2RmZiUNVfQiYtsO6NNxQmZmBvjyppmZFYiLnpmZFUZDXd4sGufpWa05z86KruFnevkbWFdpc1L6vmDp8d8lVbtrTGvHMEjSyc1se6K9jmNmZh2rIYpeSjpYlLGcBMwrehGxZ0S8v+gjW7iI2KEWxzEzs0VXt6KXQlefk3Q9MBU4WNIoSU9JGppuKF2+zxUplmdaKdJH0gnAWsAwScPSuhnpFmNI+mWKLpoq6aTcsZ9RWVxSqT9JT6fYoFtyh++ZZp0vpWOWxjQ7/e6nLCrp3vS8rlzEQm5mZu2s3v9R7kEWB7QLWZRP/4jYChgHVIoEOj19C38LYBdJW0TEJcBrwLcj4tv5xpK2Jrs92rbAdmTxQN/MHbs8LgngNOCbKTLp6Fx3mwLfJYsU+l1Khyi3DXA80JMsieJH5Q2cp2dmVj/1LnovR8STZAWpJzBS0kSy+3quV6H9TyQ9RRbr0yvtU81OwJ0R8VFEzAbuYH5uXnlcUlNangzcJOkgssSIknsj4rOIeIfs/p9rVjjemIh4KUUa3UyFtAXn6ZmZ1U+9P71Ziv8RWdbeAc01lLQ+WQpC34h4T9IQvhpd1BrlcUnLpuXvk0Ue7QWcLmnzZtpXOnflNzJt/BubmpkVSL1neiVPAjtK2ghA0vIpyidvJbIiOUvSmsAeuW0fAitW6HcEsK+k5SQtTxb7M6JCO9JxlwDWiYhhwKlkwbGtuQXaNildfQmyJIjHF7aDmZnVTr1negBExNuSBgI3p2w9gDOA53NtJkmaADwLvAKMzHUxGLhP0mv59/Ui4qk0IxyTVl0dERMkNTUzlC7AjZK6kc0+L4mI93PJ7AszFrgM2AgYBtxZrbHz9MzMaqtTRAt1BpL6ASdHxA9auo+jhczMWs/RQmZmZi3QEJc3FwcRMRwYXudhmJlZFZ7pmZlZYbjomZlZYbjomZlZYfg9vTpytJCVc/SPWcfyTM/MzAqj4YqepLUk3ZaWe0vaswX79JN0TyuP0yTpZ+3VzszMGl9DFT1JS0bEaxGxX1rVG1ho0WujJqAlxayl7czMrMG1S9FLs6FnJQ2R9LykmyT1lzRS0guStkk/oyRNkPSEpE3SvgMl3S3pEeDh1NdUSUsDZwMDJE2UNKC5Plowvl1SHxPTvisC5wM7p3W/SMcdkfL8npJUCoctbzdQ0mW5vu9JM80u6flPlTRF0i+aGYujhczM6qQ9P8iyEbA/cDjZPSh/RhatszfwG+AQYOeI+EJSf+D3zM+w2wrYIiLeLd0XMyI+l3Qm0CcijgOQtFKVPqo5GTg2IkYqC6f9lCw3b95twyQtB3wnIj6V1IMsGqhPhXYDmzlGb2DtiNgstVu5UqOIGEx2r1C6du/he8CZmdVQexa96RExBUDSNODhiAhJU8guEXYDrksFJYB8COuDEfFuC45RrY9qRgJ/knQTcEdEvFrhJtJLAZdJ6k0WHVSe8rAwLwEbSLoUuBd4oJX7m5lZB2vP9/TyeXNf5h5/SVZczwGGpZnQXiyYhfcRLVOtj2ZFxPnAv5Fl5o2UtGmFZr8A3gS2JJvhLd1Md1+w4HlbJh3jvbTvcLLE9atbMjYzM6udWn5PrxswMy0PbOE+5Tl5bekDSRumWegUSX2BTcniicr7fjUivpR0KFnMUKUxzAD+PWXmrQ1sk46xOvB5RNwu6TngxoWNy9FCZma1VctPb14AnJcy8VpabIcBPUsfZGljHwAnpQ+YTAbmAP8AJgNzJU1KHzr5b+BQSZPIimJp9lnebiQwHXgauAR4KrVbGxguaSJZwft1K8ZnZmY14Dy9OnKenplZ6zlPz8zMrAUWq3tvSjoMOLFs9ciIOLYe4zEzs8ayWBW9iLgWuLbe4zAzs8bky5tmZlYYLnpmZlYYi9Xlzc7GeXr14cw6s+IqxExP0oz05fGWtB0k6eSOHpOZmdXeYl/0JHVZeCszMyuChi56kk6RdEJavijFDyFp1xRfdECK8Zkq6Q+5/WZL+s90d5Xtc+uXlfQPSUemx4dImpzutnJDheMfKWls2n57SmJA0v7pmJMkPZbW9ZI0Jt09ZnK6KbaZmTWQhi56wAhg57TcB1hB0lJp3fPAH4BdyWJ9+kraN7VdHhgdEVtGxONp3QrA/wI3R8RVknoBZwC7RsSWfPX7fZAlMvRN258BjkjrzwS+m9bvndYdDVwcEb3TWF+t9IScp2dmVj+NXvTGA1unHL3PgFFkBWVn4H1geES8HRFfADcB30r7zQVuL+vrLuDaiLg+Pd4VGBoR7wA0E220WQqWnQIcCPRK60cCQ9KMsXT5dBTwG0mnAutFxCeVnlBEDI6IPhHRp8ty3Vp+JszMbJE1dNGLiDlkN3ceCDxBNvP7Nllg7Ywqu34aEXPL1o0EvqcKQXpVDAGOi4jNgbOYHyN0NNkscR1gvKTVIuIvZLO+T4C/S9q1FccxM7MaaOiil4wgSz5/LC0fDUwAxgC7SFo9fVjlAODRKv2cCbwHXJ4ePwLsL2k1AEmrVthnReD1dEn1wNLKFFU0OiLOBN4G1pG0AfBSRFxCNqvcoq1P2MzMOkZn+J7eCOB0YFREfCTpU2BERLwu6TSy+CEB90bEXQvp60TgGkkXRMSvJJ0LPCppLlkhHVjW/rfAaLLCNpr5uXoXpg+qCHgYmAScChwsaQ7wBvD7hT0x5+mZmdWWo4XqyNFCZmat52ghMzOzFnDRMzOzwnDRMzOzwnDRMzOzwnDRMzOzwnDRMzOzwugM39NbbDlPL+N8OzOrlZrN9CTtK6ln7vHZkvrX6viLqnz8ZmbW+dTy8ua+wLyiERFnRsRDNTz+olpg/HmSPGM2M+sE2lz0JDVJekbSVZKmSXog5dV9JYNO0g5kN2O+MOXNbShpiKT9JH1P0tBcv/0k3ZOWd5c0StJTkoZKWqHKePpKeiIdd4ykFSUtI+nalLk3QdK3U9uBki7L7XuPpH5pebakc1M/T0pas5nxD5f0X5LGAadLmp7u0YmklfKPy8bpaCEzszpZ1JleD+DyiOhFFvXzYypk0EXEE8DdwCkR0Tsi/pnr4yFgW0nLp8cDgFskrU6WZNA/IrYCxgG/rDQISUsDtwInpuP2J0s7OBaIlJJwAHCdpGUW8pyWB55M/TwGHFll/EunmKCzgOFA6c2pn6bzMKe8c0cLmZnVz6IWvekRMTEtjweaaD6DrqKUhXcfsFe6TPh9spSC7cguJ46UNBE4FFivmW42AV6PiLGpzw9SvzsBN6Z1zwIvAxsv5Dl9DtxT9pyac2tu+WrgsLR8GHDtQo5jZmY1tqjvRX2WW54LLEuWQbdvREySNBDo14J+bgGOA94FxkXEhyn37sGIOGARx1jJFyxY8POzvzkx/y7cc6l+jj4qLUTEyHTJtx/QJSKmttdgzcysfXTEBzDKM+hmpvUfMj+ap9yjwDXAkWQFEOBJ4HJJG0XEi+ny59oR8XyF/Z8DukvqGxFjJa1IdnlzRBrDI5I2BtZNbVcC/l3SEsDawDYteF7Vxl9yPfAX4JwW9OdoITOzGuuIT2+WMuhGAs/m1t8CnJI+ULJhfoeUcn4PsEf6TUS8TZZvd7OkycAoYNNKB4yIz8neC7xU0iTgQbLZ238DS6RLrbcCAyPiszS26cDTwCXAUy14Xs2OP+cmYBXg5hb0Z2ZmNeY8vXYkaT9gn4g4uCXtnadnZtZ6i5Kn5++XtRNJl5LNVPes91jMzKyyTlf0JN0JrPrchXgAAAlISURBVF+2+tSIuL8e4ymJiOPreXwzM1u4Tlf0IuKH9R6DmZl1Tk5ZMDOzwnDRMzOzwuh0lzcXJ50xWsgxQGbWmXX6mV75zaPbqc9OHYNkZmaVdfqi10E6ewySmZlV0PBFT9JBKSpooqQ/S+oi6TBJz0saA+yYazskfUG89Hh2bvnUFDE0SdL5aV2rYpDSPrulu7JMkXSNpK5p/QxJZ6UYpCmSKt49xszM6qehi56kb5DdXmzHiOhNdgPog4CzyIrdTjQT7FrWzx7APsC2KTLogrSpVTFIKZZoCDAgxRUtCRyTO9Q7KQbpCuDkZsbiPD0zszpp6KIH7AZsDYxN8UK7Ab8AhkfE2+mem7dW6yDpD1wbER8DRMS7aX2rYpDIIoym5256fR3wrdz2O9LvZiOJnKdnZlY/jV70BFyXZly9I2ITYFCV9vMig1KCwtIL6X8IcFyatZ3FghFDbVGKWlpYJJGZmdVBoxe9h4H9JH0NQNKqwARgF0mrpfii/XPtZ5DNDCF7X26ptPwgcJik5XL9wFdjkEqaixF6DmiStFF6fDBZLJKZmXUCDT0biYinJZ0BPJBmbnOAY8lme6OA94GJuV2uAu5K8UL3kUJeI+I+Sb2BcZI+B/4O/Ib5MUhvp9+lQncLcJWkE4B5H4yJiE8lHQYMTSnvY4Er2/r8nKdnZlZbjhaqI0cLmZm13qJECzX65U0zM7N246JnZmaF4aJnZmaF4aJnZmaF4aJnZmaF4aJnZmaF0dDf01vcNWqenjPzzGxx5ZleM5Tx+TEzW4z4P+o5kpokPSfpemAq8D8pEWGapLNy7fpKeiJFEo2RtGKKPLowRRVNlvTz+j0TMzOrxJc3v6oHcGhEPClp1Yh4V1IX4GFJWwDPkiU7DIiIsZJWAj4BjgBmRUTflLE3UtIDETG9bs/EzMwW4KL3VS9HxJNp+SeSjiI7T93JsvsCeD0ixgJExAcAknYHtsiF2HYjK6ALFL3U31EAXVZao4OfipmZ5bnofdVHAJLWJwuC7RsR70kaQvXoIQHHR8T91TqPiMHAYICu3Xv4xqdmZjXk9/SatxJZAZwlaU1gj7T+OaC7pL4A6f28JYH7gWNSTBGSNpa0fB3GbWZmzfBMrxkRMUnSBLL38F4BRqb1n0saAFwqaVmy9/P6A1eTpaU/JUlkcUX7VjuGo4XMzGrL0UJ15GghM7PWc7SQmZlZC7jomZlZYbjomZlZYbjomZlZYbjomZlZYbjomZlZYfh7enVU62ghRwaZWdF5pmdmZoXhomdmZoXholeFpOUl3Zty86ZKGiBpa0mPShov6X5J3SUtmXL0+qX9zpN0bp2Hb2ZmZfyeXnXfA16LiO8DSOoG/APYJyLeTvfgPDciDpc0ELhN0vFpv20rdehoITOz+nHRq24K8J+S/gDcA7wHbAY8mN1Tmi7A6wARMU3SDand9hHxeaUOHS1kZlY/LnpVRMTzkrYC9gT+H/AIMC0itm9ml82B94Gv1WiIZmbWCn5PrwpJawEfR8SNwIVklyzXkLR92r6UpF5p+UfAqsC3yGKHVq7TsM3MrBme6VW3OXChpC+BOcAxwBfAJen9vSWB/5L0JnA+sFtEvCLpMuBi4NCqnTtPz8ysplz0qoiI+8kS0ct9q8K6jXP7XdJhgzIzszbz5U0zMysMFz0zMysMRfhT8/Ui6UPguXqPowGsDrxT70E0AJ+HjM/DfD4XmfLzsF5EtOmLzn5Pr76ei4g+9R5EvUka5/Pg81Di8zCfz0WmPc+DL2+amVlhuOiZmVlhuOjV1+B6D6BB+DxkfB4yPg/z+Vxk2u08+IMsZmZWGJ7pmZlZYbjomZlZYbjo1YGk70l6TtKLkk6r93g6kqR1JA2T9LSkaZJOTOtXlfSgpBfS71XSekm6JJ2bySnlYrEhqYukCZLuSY/XlzQ6Pd9bJS2d1ndNj19M25vqOe72JmllSbdJelbSM5K2L+JrQtIv0r+LqZJulrRMEV4Tkq6R9Jakqbl1rf77Szo0tX9BUtV7HZe46NWYpC7A5cAeQE/gAEk96zuqDvUF8B8R0RPYDjg2Pd/TgIcjogfwcHoM2XnpkX6OAq6o/ZA71InAM7nHfwAuioiNyPIaj0jrjwDeS+svSu0WJxcD90XEpsCWZOekUK8JSWsDJwB9ImIzsnzOn1KM18QQsrDtvFb9/SWtCvyOLP1mG+B3pUJZVUT4p4Y/wPbA/bnHvwZ+Xe9x1fD53wV8h+xONN3Tuu5kX9QH+DNwQK79vHad/Qf4evrHvCtZ2LDI7jKxZPlrg+xG59un5SVTO9X7ObTTeegGTC9/PkV7TQBrA6+QRZItmV4T3y3KawJoAqa29e8PHAD8Obd+gXbN/XimV3ulF3rJq2ndYi9djvkmMBpYMyJeT5veANZMy4vz+fkv4FfAl+nxasD7EfFFepx/rvPOQ9o+K7VfHKwPvA1cmy71Xi1peQr2moiImcAfgf8DXif7G4+nmK8JaP3fv02vCxc9qwlJKwC3AydFxAf5bZH9b9pi/d0ZST8A3oqI8fUeSwNYEtgKuCIivgl8xPxLWUBhXhOrAPuQ/U/AWsDyfPWSXyF15N/fRa/2ZgLr5B5/Pa1bbElaiqzg3RQRd6TVb0rqnrZ3B95K6xfX87MjsLekGcAtZJc4LwZWllS6B27+uc47D2l7N+BftRxwB3oVeDUiRqfHt5EVwaK9JvoD0yPi7YiYA9xB9jop4msCWv/3b9PrwkWv9sYCPdIntJYme+P67jqPqcNIEvA/wDMR8afcpruZnyx/KNl7faX1h6RPbG0HzMpd8ui0IuLXEfH1iGgi+5s/EhEHAsOA/VKz8vNQOj/7pfaLxcwnIt4AXpG0SVq1G/A0BXtNkF3W3E7ScunfSek8FO41kbT2738/sLukVdKseXcqh34vqN5vZhbxB9gTeB74J3B6vcfTwc91J7LLFJOBielnT7L3Ih4GXgAeAlZN7UX26dZ/AlPIPtlW9+fRzuekH3BPWt4AGAO8CAwFuqb1y6THL6btG9R73O18DnoD49Lr4m/AKkV8TQBnAc8CU4EbgK5FeE0AN5O9jzmHbOZ/RFv+/sDh6Xy8CBzWkmP7NmRmZlYYvrxpZmaF4aJnZmaF4aJnZmaF4aJnZmaF4aJnZmaF4aJnZmaF4aJnZmaF8f8BffLzTNmusxEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# question 7: plot feature importance\n",
    "feature_imp = xgb.train(param, train_initial, num_boost_round = df_cv.shape[0])\n",
    "imp_dict = feature_imp.get_fscore()\n",
    "df_imp = pd.DataFrame.from_dict(imp_dict, orient = 'index')\n",
    "df_imp.columns = ['importance']\n",
    "df_imp = df_imp['importance'].sort_values()\n",
    "df_imp.plot.barh()\n",
    "plt.title('importance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7789647187972366"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# question 8: analyze auc score on Test Data,\n",
    "test_initial = xgb.DMatrix(x_test, y_test)\n",
    "predict_result = feature_imp.predict(test_initial)\n",
    "roc_auc_score(y_test, predict_result.round())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
